{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 推荐系统"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'sys' (built-in)>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# -*- coding:utf-8 -*-\n",
    "#import sys\n",
    "#reload(sys)\n",
    "#sys.setdefaultencoding(\"utf-8\")\n",
    "###以上代码适用于python2，Python3替换成如下的代码：\n",
    "import importlib\n",
    "import sys\n",
    "importlib.reload(sys)\n",
    "#sys.setdefaultencoding(\"utf-8\")  ###Python3字符串默认编码unicode, 所以sys.setdefaultencoding也不存在了"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T05:34:54.624167Z",
     "start_time": "2017-09-26T05:34:46.420964Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from collections import defaultdict\n",
    "import scipy.sparse as ss\n",
    "\n",
    "#保存数据\n",
    "import pickle\n",
    "import scipy.io as sio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据\n",
    "#用户(10万)、歌曲（3万）、播放次数、歌曲元数据\n",
    "用户(5000)、歌曲（5000）、播放次数、歌曲元数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('triplet_dataset_sub_song_merged.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 隐式反馈 --> 打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#每个用户的总播放次数\n",
    "triplet_train_sum_df = train[['user','listen_count']].groupby('user').sum().reset_index()\n",
    "triplet_train_sum_df.rename(columns={'listen_count':'total_listen_count'},inplace=True)\n",
    "\n",
    "#每首歌曲的播放比例\n",
    "train = pd.merge(train,triplet_train_sum_df)\n",
    "train['fractional_play_count'] = train['listen_count']/train['total_listen_count']\n",
    "del triplet_train_sum_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>listen_count</th>\n",
       "      <th>title</th>\n",
       "      <th>release</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year</th>\n",
       "      <th>total_listen_count</th>\n",
       "      <th>fractional_play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAFTRR12AF72A8D4D</td>\n",
       "      <td>1</td>\n",
       "      <td>Harder Better Faster Stronger</td>\n",
       "      <td>Discovery</td>\n",
       "      <td>Daft Punk</td>\n",
       "      <td>2007</td>\n",
       "      <td>861</td>\n",
       "      <td>0.001161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAIILB12A58A776F7</td>\n",
       "      <td>3</td>\n",
       "      <td>Phantom Part 1.5 (Album Version)</td>\n",
       "      <td>A Cross The Universe</td>\n",
       "      <td>Justice</td>\n",
       "      <td>0</td>\n",
       "      <td>861</td>\n",
       "      <td>0.003484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAJJDS12A8C13A3FB</td>\n",
       "      <td>1</td>\n",
       "      <td>I Got Mine</td>\n",
       "      <td>Attack &amp; Release</td>\n",
       "      <td>The Black Keys</td>\n",
       "      <td>2008</td>\n",
       "      <td>861</td>\n",
       "      <td>0.001161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAMDXO12A8C131E2F</td>\n",
       "      <td>2</td>\n",
       "      <td>Pogo</td>\n",
       "      <td>Idealism</td>\n",
       "      <td>Digitalism</td>\n",
       "      <td>2007</td>\n",
       "      <td>861</td>\n",
       "      <td>0.002323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAMPRJ12A8AE45F38</td>\n",
       "      <td>20</td>\n",
       "      <td>Rorol</td>\n",
       "      <td>Identification Parade</td>\n",
       "      <td>Octopus Project</td>\n",
       "      <td>2002</td>\n",
       "      <td>861</td>\n",
       "      <td>0.023229</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user                song  listen_count  \\\n",
       "0  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAFTRR12AF72A8D4D             1   \n",
       "1  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAIILB12A58A776F7             3   \n",
       "2  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAJJDS12A8C13A3FB             1   \n",
       "3  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAMDXO12A8C131E2F             2   \n",
       "4  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAMPRJ12A8AE45F38            20   \n",
       "\n",
       "                              title                release      artist_name  \\\n",
       "0     Harder Better Faster Stronger              Discovery        Daft Punk   \n",
       "1  Phantom Part 1.5 (Album Version)   A Cross The Universe          Justice   \n",
       "2                        I Got Mine       Attack & Release   The Black Keys   \n",
       "3                              Pogo               Idealism       Digitalism   \n",
       "4                             Rorol  Identification Parade  Octopus Project   \n",
       "\n",
       "   year  total_listen_count  fractional_play_count  \n",
       "0  2007                 861               0.001161  \n",
       "1     0                 861               0.003484  \n",
       "2  2008                 861               0.001161  \n",
       "3  2007                 861               0.002323  \n",
       "4  2002                 861               0.023229  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 子集抽取\n",
    "要计算用户播放过的歌曲与所有30万首歌的相似度太慢，可考虑只推荐最流行的1000首歌"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "song_count_df = pd.read_csv('song_playcount_df.csv')\n",
    "song_count_subset = song_count_df.head(n=1000)\n",
    "song_subset = list(song_count_subset.song)\n",
    "trian_sub = train[train.song.isin(song_subset)]\n",
    "\n",
    "del train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(211686, 9)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trian_sub.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "20万条播放记录"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "items = list(song_count_subset['song'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "users = list(trian_sub['user'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 事先计算好倒排表，比实时查询数据库快\n",
    "用户和item重新建索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of Users :4974\n",
      "number of Songs :1000\n"
     ]
    }
   ],
   "source": [
    "n_users = len(users)\n",
    "n_items = len(items)\n",
    "\n",
    "print(\"number of Users :%d\" % n_users)\n",
    "print(\"number of Songs :%d\" % n_items)\n",
    "\n",
    "#倒排表\n",
    "#统计每个用户播放过的歌曲   / 播放每个歌曲的用户\n",
    "user_items = defaultdict(set)\n",
    "item_users = defaultdict(set)\n",
    "\n",
    "#用户-物品关系矩阵表，稀疏矩阵，\n",
    "user_items_scores = ss.dok_matrix((n_users, n_items))\n",
    "\n",
    "#重新编码用户索引字典\n",
    "user_index = dict()\n",
    "item_index = dict()\n",
    "for i, u in enumerate(users):\n",
    "    user_index[u] = i\n",
    "\n",
    "\n",
    "#重新编码活动索引字典    \n",
    "for i, e in enumerate(items):\n",
    "    item_index[e] = i\n",
    "\n",
    "n_records = trian_sub.shape[0]\n",
    "for i in range(n_records):\n",
    "    user_index_i = user_index[trian_sub.iloc[i]['user'] ] #用户\n",
    "    item_index_i = item_index[trian_sub.iloc[i]['song'] ]#歌曲\n",
    "    \n",
    "    user_items[user_index_i].add(item_index_i)    #该用户的歌曲\n",
    "    item_users[item_index_i].add(user_index_i)    #播放该歌曲的用户\n",
    "        \n",
    "    score = trian_sub.iloc[i]['fractional_play_count']  #播放次数的比例\n",
    "    user_items_scores[user_index_i, item_index_i] = score\n",
    "\n",
    "  \n",
    "#倒排表\n",
    "pickle.dump(user_items, open(\"user_items.pkl\", 'wb'))\n",
    "pickle.dump(item_users, open(\"item_users.pkl\", 'wb'))\n",
    "\n",
    "#保存用户-物品关系矩阵R，以备后用\n",
    "sio.mmwrite(\"user_items_scores\", user_items_scores)\n",
    "\n",
    "\n",
    "#保存用户索引表\n",
    "pickle.dump(user_index, open(\"user_index.pkl\", 'wb'))\n",
    "#保存活动索引表\n",
    "pickle.dump(item_index, open(\"item_index.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 事先计算好所有item之间的相似性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i=:0 \n",
      "i=:10 \n",
      "i=:20 \n",
      "i=:30 \n",
      "i=:40 \n",
      "i=:50 \n",
      "i=:60 \n",
      "i=:70 \n",
      "i=:80 \n",
      "i=:90 \n",
      "i=:100 \n",
      "i=:110 \n",
      "i=:120 \n",
      "i=:130 \n",
      "i=:140 \n",
      "i=:150 \n",
      "i=:160 \n",
      "i=:170 \n",
      "i=:180 \n",
      "i=:190 \n",
      "i=:200 \n",
      "i=:210 \n",
      "i=:220 \n",
      "i=:230 \n",
      "i=:240 \n",
      "i=:250 \n",
      "i=:260 \n",
      "i=:270 \n",
      "i=:280 \n",
      "i=:290 \n",
      "i=:300 \n",
      "i=:310 \n",
      "i=:320 \n",
      "i=:330 \n",
      "i=:340 \n",
      "i=:350 \n",
      "i=:360 \n",
      "i=:370 \n",
      "i=:380 \n",
      "i=:390 \n",
      "i=:400 \n",
      "i=:410 \n",
      "i=:420 \n",
      "i=:430 \n",
      "i=:440 \n",
      "i=:450 \n",
      "i=:460 \n",
      "i=:470 \n",
      "i=:480 \n",
      "i=:490 \n",
      "i=:500 \n",
      "i=:510 \n",
      "i=:520 \n",
      "i=:530 \n",
      "i=:540 \n",
      "i=:550 \n",
      "i=:560 \n",
      "i=:570 \n",
      "i=:580 \n",
      "i=:590 \n",
      "i=:600 \n",
      "i=:610 \n",
      "i=:620 \n",
      "i=:630 \n",
      "i=:640 \n",
      "i=:650 \n",
      "i=:660 \n",
      "i=:670 \n",
      "i=:680 \n",
      "i=:690 \n",
      "i=:700 \n",
      "i=:710 \n",
      "i=:720 \n",
      "i=:730 \n",
      "i=:740 \n",
      "i=:750 \n",
      "i=:760 \n",
      "i=:770 \n",
      "i=:780 \n",
      "i=:790 \n",
      "i=:800 \n",
      "i=:810 \n",
      "i=:820 \n",
      "i=:830 \n",
      "i=:840 \n",
      "i=:850 \n",
      "i=:860 \n",
      "i=:870 \n",
      "i=:880 \n",
      "i=:890 \n",
      "i=:900 \n",
      "i=:910 \n",
      "i=:920 \n",
      "i=:930 \n",
      "i=:940 \n",
      "i=:950 \n",
      "i=:960 \n",
      "i=:970 \n",
      "i=:980 \n",
      "i=:990 \n"
     ]
    }
   ],
   "source": [
    "similarity_matrix = np.matrix(np.zeros(shape=(n_items, n_items)), float)\n",
    "\n",
    "for i in range(n_items):\n",
    "    #得到item i的特征表示：Calculate unique users of item i\n",
    "    users_i = item_users[i]\n",
    "    similarity_matrix[i,i] = 1.0\n",
    "    \n",
    "    if(i % 10 == 0):\n",
    "        print(\"i=:%d \" % (i))\n",
    "\n",
    "        for j in range(i+1,n_items):   #items by user    \n",
    "            #得到item j的特征表示：Get unique users of item j\n",
    "            users_j = item_users[j]\n",
    "                    \n",
    "            #Calculate intersection of listeners of songs i and j\n",
    "            #计算item i与item j的相似度\n",
    "            users_intersection = users_i.intersection(users_j)\n",
    "                \n",
    "            #Calculate cooccurence_matrix[i,j] as Jaccard Index\n",
    "            if len(users_intersection) != 0:\n",
    "                #Calculate union of listeners of songs i and j\n",
    "                users_union = users_i.union(users_j)\n",
    "                similarity_matrix[j,i] = float(len(users_intersection))/float(len(users_union))\n",
    "            else:\n",
    "                similarity_matrix[j,i] = 0\n",
    "                    \n",
    "        similarity_matrix[i,j] = similarity_matrix[j,i]\n",
    "        pickle.dump(similarity_matrix, open(\"items_similarity.pkl\", 'wb'))\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {
    "height": "153px",
    "width": "160px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": true,
   "toc_position": {
    "height": "691px",
    "left": "0px",
    "right": "1405px",
    "top": "106px",
    "width": "303px"
   },
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
